import requests
from bs4 import BeautifulSoup

if __name__ == '__main__':
    url = "https://www.shicimingju.com/book/sanguoyanyi.html"
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    }
    response = requests.get(url=url, headers=headers)
    response.encoding = "utf-8"
    page_text = response.text
    # 在首页中解析出章节的标题和详情页的url
    # 实例化BeautifulSoup对象，需要将页面源码数据加载到该对象中
    soup = BeautifulSoup(page_text, "lxml")
    # 解析章节标题和详情页url
    url_list = []
    li_list = soup.select(".book-mulu > ul > li")
    print(li_list[0].text)
    # fp = open("./sanguo.txt", "w", encoding="utf-8")
    # for li in li_list:
    #     detail_url = "http://www.shicimingju.com" + li.a["href"]
    #     title = li.a.string
    #     # 对详情页发起请求
    #     detail_page_text = requests.get(url=detail_url, headers=headers).text
    #     # 解析出详情页中相关的章节内容
    #     detail_soup = BeautifulSoup(detail_page_text, "lxml")
    #     div_tag = detail_soup.find("div", class_="chapter_content")
    #     content = div_tag.text
    #     fp.write(title+":"+content+"\n")
    #     print(title,"爬取成功")
